# -*- coding: utf-8 -*-
"""
__title__ = ''
__author__ = 'Yang'
__mtime__ = '2019/1/17 17:16'
# code is far away from bugs with the god animal protecting
    I love animals. They taste delicious.
              ┏┓      ┏┓
            ┏┛┻━━━┛┻┓
            ┃      ☃      ┃
            ┃  ┳┛  ┗┳  ┃
            ┃      ┻      ┃
            ┗━┓      ┏━┛
                ┃      ┗━━━┓
                ┃  神兽保佑    ┣┓
                ┃　永无BUG！   ┏┛
                ┗┓┓┏━┳┓┏┛
                  ┃┫┫  ┃┫┫
                  ┗┻┛  ┗┻┛
"""

import re
import requests
import time
from bs4 import BeautifulSoup
import json
import random

user_agent = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/64.0',
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/65.0',
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
              'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2767.400',
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134']
cookie = [
    'xtest=5531.cf6b6759; ipLoc-djd=1-72-2799-0; __jda=122270672.15477843634821916159958.1547784363.1547795348.1547859645.3; __jdv=122270672|direct|-|none|-|1547784363487; shshshfp=b6c2c2c2c9a00226892f48d193fc7b56; shshshfpa=e50dfc2d-35b9-5dba-681c-bec228f9f2ea-1547784363; __jdu=15477843634821916159958; qrsc=3; shshshfpb=rxdelY1j6aCconIE6b3rgFA%3D%3D; PCSYCityID=country_2468; 3AB9D23F7A4B3C9B=J6GH2L6XJQWKKXWD4EZQOJU3ILBSHVRYSG64QXLQAKH3SMKLUDXE3DWN5BPQUUXNJLIKZXNKTJ3MTCZFIWF7GNLWOE; user-key=26beea7b-02a7-47d9-8a5f-1ff7d7113d68; cn=0; __jdb=122270672.4.15477843634821916159958|3.1547859645; __jdc=122270672; shshshsID=6d2224cda48f5b1e8fd608ec704f4dc3_2_1547859676845; rkv=V0400; wlfstk_smdl=tzm0ktdtywi3kszwrv4er4fifou4omxh; TrackID=1kIbg1pcWI4LF0x0UxmXNp8PXaZU6ZyxJAjKHgKRh-oK7OSSxXtoUkKcW0SATTtSlksen0Sfnbtd0Ck9QmSBrFWRHnP1p90e1TYO8QgRwJMCaEkYBI7tesISuA3elttgw; thor=D1A72F2D0BF9CB753C84DD47AF3C84DEE863BC08DF59DBDD33D8F594A81BD41CF9E76A07607FD64BCB3CAE7BBFDC05D90AB2809E2EDE293064A9B5C6023B6C174285F814BE47485E584F542CDD5011441203CE6FCB554A2ABBFDE74E2B829DABA83342E306B799FAED95C04FDABFACC4C4AB7ED82E52AD19BEAEE119CB9773AC7BA460B9FB266C9BDB6063C9EDA3FE8382A2D28FA248D8CDC6779330EDF803A7; pinId=8EMaZEroy6EI7FsztX-BnrV9-x-f3wj7; pin=jd_47d430c34eabe; unick=jd_47d430c34eabe; ceshi3.com=103; _tp=9FnEhPpJaUQq%2F048DEdFRJ2ce8lD%2B5CH3iPtlc7O9WE%3D; _pst=jd_47d430c34eabe',
    '__jdv=122270672|baidu|-|organic|jd|1547716306389; PCSYCityID=1132; shshshfpa=38e7d6aa-5f8c-754a-3e7d-f188ecdfcf1d-1547716308; shshshfpb=tzEKAh4R7RIOuiKLC66gdng%3D%3D; mt_xid=V2_52007VwMWV1teUl0dSB5eDG4DEVFcWlJdF0EabA03BEZWClEARhhMGAwZYlMaBkFQWghKVUlbAWQAEQVdUABSF3kaXQVvHxNSQVhUSx9PElwFbAcSYl1oUmodTBtfAWUEEFttWlZbHQ%3D%3D; xtest=9745.cf6b6759; ipLoc-djd=1-72-2799-0; __jdu=15477163063881220316992; qrsc=3; pinId=tbYlNaQwN7W3y7kTB1CvrrV9-x-f3wj7; pin=jd_53da1fdc41287; unick=Cube_V; _tp=3ZWp5j7yhBAXRgTCt9jDuv4YSIUPtqsstNdf0dYKp8A%3D; _pst=jd_53da1fdc41287; user-key=c48303d0-6b56-4945-babe-16f91807bb2c; cn=0; shshshfp=5ca3ef97c626e63b82c69a4477db0ef4; __jda=122270672.15477163063881220316992.1547716306.1547815563.1547858365.11; __jdc=122270672; rkv=V0000; ceshi3.com=103; 3AB9D23F7A4B3C9B=BNF4ERVPOLB3SN56DAATXREECV627TWVSCZSRREYLM5MVOKTJAK7KCO6JYD7T67DKKT6GVBZD4WKP3G2HBEB5ZITZY; wlfstk_smdl=np6px8iw5uwlfw26anp7xjgmajm8wksk; TrackID=1ML0sFCitrYAuIxc7RP-KsQ3ET_T_nq15aPErC5njDcwprVrCwycint8oo2mbLD23afNSopDMeJmyLQvGI4XWq04nhozt2Zmn9sizMO-jTT8; thor=72332D64FED93EC46856BEB79C3B95F230217F7CFA500AD5087564A62697B22B0CB8B135DAE1E7A59DF9E0B638F539C3738311F10DD49A59561CC5F6E86E2ACFE2719760BEC75215921728921C78F31082B2A8E1A56FB73D2C8666B15A206B54B73B8042FD915CAAB10E77BA791BDD25E5396AD74D58DF0B637A4638AF6DCBDEFFE5CF45DA5857F87DF9F1F16ADF4D9B5C3C36FC0F0DFDDD2DC3C6148B4DC07F; __jdb=122270672.10.15477163063881220316992|11.1547858365; shshshsID=04828825de82e074f55246d0033769d3_6_1547859568215',
    ]
headers_get = {
    'authority': 'search.jd.com',
    'method': 'GET',
    'path': '/Search?keyword=%E5%8D%8E%E4%B8%BAmate20&enc=utf-8&wq=%E5%8D%8E%E4%B8%BAmate20&pvid=34e7d898da614354954657e5dae76f3e',
    'scheme': 'https',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'accept-encoding': 'gzip, deflate, br',
    'Cookie': '__jdv=122270672|baidu|-|organic|jd|1547716306389; PCSYCityID=1132; shshshfpa=38e7d6aa-5f8c-754a-3e7d-f188ecdfcf1d-1547716308; shshshfpb=tzEKAh4R7RIOuiKLC66gdng%3D%3D; mt_xid=V2_52007VwMWV1teUl0dSB5eDG4DEVFcWlJdF0EabA03BEZWClEARhhMGAwZYlMaBkFQWghKVUlbAWQAEQVdUABSF3kaXQVvHxNSQVhUSx9PElwFbAcSYl1oUmodTBtfAWUEEFttWlZbHQ%3D%3D; xtest=9745.cf6b6759; ipLoc-djd=1-72-2799-0; __jdu=15477163063881220316992; qrsc=3; TrackID=1vTLijo-GakWze5o4RbGoxCt9agCd0XlhGGrIUFWcS6GPvIwA-u4q46cLS7xusOFxY714F4dc539XdZ325ZiA8gYujhRXXitzPBqfu35j4jo; pinId=tbYlNaQwN7W3y7kTB1CvrrV9-x-f3wj7; pin=jd_53da1fdc41287; unick=Cube_V; _tp=3ZWp5j7yhBAXRgTCt9jDuv4YSIUPtqsstNdf0dYKp8A%3D; _pst=jd_53da1fdc41287; user-key=c48303d0-6b56-4945-babe-16f91807bb2c; cn=0; __jda=122270672.15477163063881220316992.1547716306.1547807470.1547815563.10; __jdc=122270672; shshshfp=5ca3ef97c626e63b82c69a4477db0ef4; rkv=V0000; 3AB9D23F7A4B3C9B=BNF4ERVPOLB3SN56DAATXREECV627TWVSCZSRREYLM5MVOKTJAK7KCO6JYD7T67DKKT6GVBZD4WKP3G2HBEB5ZITZY; __jdb=122270672.3.15477163063881220316992|10.1547815563; shshshsID=a7d8af4766c718c776fedd8443b0a2bf_3_1547815621877',
    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,la;q=0.7',
    'referer': 'https://passport.jd.com/new/login.aspx?ReturnUrl=https%3A%2F%2Fsearch.jd.com%2FSearch%3Fkeyword%3D%25E5%258D%258E%25E4%25B8%25BAmate20%26enc%3Dutf-8%26wq%3D%25E5%258D%258E%25E4%25B8%25BAmate20%26pvid%3D34e7d898da614354954657e5dae76f3e',
    'upgrade-insecure-requests': '1',
}
# user-agent
headers_xhl = {
    'authority': 'search.jd.com',
    'method': 'GET',
    'path': '/s_new.php?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA',
    'scheme': 'https',
    'referer': 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=3&s=58&click=0',
    'x-requested-with': 'XMLHttpRequest',
    'Cookie': 'qrsc=3; pinId=RAGa4xMoVrs; xtest=1210.cf6b6759; ipLocation=%u5E7F%u4E1C; _jrda=5; TrackID=1aUdbc9HHS2MdEzabuYEyED1iDJaLWwBAfGBfyIHJZCLWKfWaB_KHKIMX9Vj9_2wUakxuSLAO9AFtB2U0SsAD-mXIh5rIfuDiSHSNhZcsJvg; shshshfpa=17943c91-d534-104f-a035-6e1719740bb6-1525571955; shshshfpb=2f200f7c5265e4af999b95b20d90e6618559f7251020a80ea1aee61500; cn=0; 3AB9D23F7A4B3C9B=QFOFIDQSIC7TZDQ7U4RPNYNFQN7S26SFCQQGTC3YU5UZQJZUBNPEXMX7O3R7SIRBTTJ72AXC4S3IJ46ESBLTNHD37U; ipLoc-djd=19-1607-3638-3638.608841570; __jdu=930036140; user-key=31a7628c-a9b2-44b0-8147-f10a9e597d6f; areaId=19; __jdv=122270672|direct|-|none|-|1529893590075; PCSYCityID=25; mt_xid=V2_52007VwsQU1xaVVoaSClUA2YLEAdbWk5YSk9MQAA0BBZOVQ0ADwNLGlUAZwQXVQpaAlkvShhcDHsCFU5eXENaGkIZWg5nAyJQbVhiWR9BGlUNZwoWYl1dVF0%3D; __jdc=122270672; shshshfp=72ec41b59960ea9a26956307465948f6; rkv=V0700; __jda=122270672.930036140.-.1529979524.1529984840.85; __jdb=122270672.1.930036140|85.1529984840; shshshsID=f797fbad20f4e576e9c30d1c381ecbb1_1_1529984840145'
}

price_url_1 = 'https://p.3.cn/prices/mgets?callback=jQuery9992614&type=1&area=1_72_2799_0&pdtk=&pduid=15477163063881220316992&pdpin=jd_53da1fdc41287&pin=jd_53da1fdc41287&pdbp=0&skuIds=J_{0}&ext=11100000&source=item-pc'

comment_url = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv1194&productId={0}&score=0&sortType=5&page=0&pageSize=10'


def request_url(url, head):
    page = requests.get(url, headers=head)
    page.encoding = page.apparent_encoding
    return page.text


def parse_html(html_text, jd_data_file):
    rex = re.compile('<li.class="gl-item".data-sku=".*?".data-spu=".*?".data-pid="(.*?)"')
    pro_id_list = re.findall(rex, html_text)
    soup = BeautifulSoup(html_text, 'lxml')
    shops = soup.select('span.J_im_icon > a')
    shop_name_list = [i.get('title') for i in shops]
    print(pro_id_list)
    print(len(pro_id_list))
    for k, v in zip(pro_id_list, shop_name_list):
        start_time = time.time()
        flag = {}
        # 商品id
        flag['goods_id'] = k
        # 店铺名称
        flag['shop_name'] = v
        try:
            index = random.randint(0, len(user_agent) - 1)
            headers_get['user-agent'] = user_agent[index]
            headers_xhl['user-agent'] = user_agent[index]
            pro_info_url = 'https://item.jd.com/{0}.html'.format(k)
            page_html = request_url(pro_info_url, headers_get)
            page_soup = BeautifulSoup(page_html, 'lxml')
            # 商品名
            flag['pro_name'] = page_soup.select('div > div.itemInfo-wrap > div.sku-name')[0].text.replace('\n',
                                                                                                          '').replace(
                ' ', '')
            # 价格
            flag['price'] = json.loads(request_url(price_url_1.format(k), headers_xhl)[len('jQuery9992614(['):-4])['p']
            # 店铺信息
            if '自营' not in v:
                if '旗舰' not in v and '专营' not in v:
                    flag['shop_star'] = re.findall('<div.class="star-gray".title="(.*?)">', page_html)[0]
                else:
                    flag['shop_star'] = ' '
                shop_score = re.findall(
                    '<div.class="score-detail.*?"><em.title="(.*?)".class="number">.*?</em></div>',
                    page_html)
                flag['shop_goods_score'] = shop_score[0]
                flag['shop_logistics_score'] = shop_score[1]
                flag['shop_after_sale_score'] = shop_score[2]
            else:
                flag['shop_star'] = '自营'
                flag['shop_goods_score'] = '自营'
                flag['shop_logistics_score'] = '自营'
                flag['shop_after_sale_score'] = '自营'
            comment_json = \
                json.loads(request_url(comment_url.format(k), headers_xhl)[len('fetchJSON_comment98vv1194('):-2])[
                    'productCommentSummary']
            # 总评
            flag['comment_count'] = comment_json['commentCountStr']
            # 好评
            flag['good_count'] = comment_json['goodCountStr']
            # 好评度
            flag['good_rate'] = comment_json['goodRate']
            print(flag['goods_id'])
            flag_str = '{},{},{},{},{},{},{},{},{},{},{}'.format(flag['goods_id'], flag['shop_name'], flag['pro_name'],
                                                                 flag['price'], flag['shop_star'],
                                                                 flag['shop_goods_score'],
                                                                 flag['shop_logistics_score'],
                                                                 flag['shop_after_sale_score'],
                                                                 flag['comment_count'], flag['good_count'],
                                                                 flag['good_rate'])
            jd_data_file.write(flag_str + '\n')
            jd_data_file.flush()
            print('page time: ', time.time() - start_time)
        except:
            print('error:!!!!!!!!!!!!!!!!!!!!', flag['goods_id'])
            with open('error.txt', 'a', encoding='utf-8') as f:
                f.write(flag['goods_id'] + '\n')


url = 'https://search.jd.com/Search?keyword=%E5%8D%8E%E4%B8%BAmate20&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&bs=1&suggest=1.his.0.0&ev=exbrand_%E5%8D%8E%E4%B8%BA%EF%BC%88HUAWEI%EF%BC%89%5E&page={0}&s={1}&click=0'

url_back_30 = 'https://search.jd.com/s_new.php?keyword=%E5%8D%8E%E4%B8%BAmate20&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&bs=1&suggest=1.his.0.0&ev=exbrand_%E5%8D%8E%E4%B8%BA%EF%BC%88HUAWEI%EF%BC%89%5E&page={0}&s={1}&scrolling=y&log_id={2}&tpl=3_M'
if __name__ == '__main__':
    for i in range(29, 48):
        with open('jd_data.csv', 'a', encoding='utf-8') as jd_data_file:
            print(i, "---------------------------------------前30---------------------------------------")
            headers_get['user-agent'] = user_agent[random.randint(0, len(user_agent) - 1)]
            u = url.format(i * 2 - 1, i * 60 - 60)
            print(u)
            parse_html(request_url(u, headers_get), jd_data_file)
            print(i, "---------------------------------------后30---------------------------------------")
            headers_xhl['user-agent'] = user_agent[random.randint(0, len(user_agent) - 1)]
            parse_html(request_url(url_back_30.format(i * 2 - 1, i * 60 - 60, '%0.5f' % time.time()), headers_xhl),
                       jd_data_file)
